# -*- coding: utf-8 -*-
"""
Created on Wed Aug  8 09:32:22 2018

@author: sn5tfg
"""


import os
import re
import sys
import commands
import codecs
import re
from xml.dom.minidom import parse, parseString

import time





import parseAndPushSettings as settings

xmlFolders=settings.xmlFolders

encList=settings.encList
targetEnc=settings.targetEnc
logFolder=settings.logFolder




import logging
logging.basicConfig(level=logging.DEBUG,
                format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                datefmt='%Y-%m-%d %H:%M:%S', 
                filename=os.path.join(logFolder,'parseAndPush_21.log'))

import traceback




    

class ParseAndPush :
    
    def parseXML(self)  :

                
        print '############################################  targetEnc:', targetEnc
        xmlEncoding = '<?xml version="1.0" encoding="targetEnc"?>'.replace("targetEnc", targetEnc) # xmlEncoding用于替换xml文件开头的<?xml...?>声明(和encode(targetEnc)是对应的)


        xmlFileFullPathNameList = []
        for xmlFolderSingleFolder in xmlFolders : # 遍历存有xml的多个文件夹
             for xmlFileName in os.listdir(xmlFolderSingleFolder) : # 循环出每个文件夹下的文件名（非全路径）
                  xmlFileFullPathName = os.path.join(xmlFolderSingleFolder,xmlFileName) # 将文件名转成全路径名                  
                  xmlFileFullPathNameList.append(xmlFileFullPathName) # 添加到全路径名List

        for fileName in xmlFileFullPathNameList:
            print fileName
            try:          
				# 1. 把xml文件解析成dom
				print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>__CURRENT_FILE: %s " % (fileName)
				logging.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>__CURRENT_FILE: %s " % (fileName))
				
				file_xml = open(fileName,"r").read()            
				# minidom只支持utf-8文件的解析，故有如下操作：将<?xml...?>声明中的encoding，改为utf-8
				file_xml = re.sub(r'<\?xml ([\s\S]*)\?>', xmlEncoding, file_xml) #([\s\S]*)可用来匹配任意字符，遇到"\n"不会有影响（注：(.*?)受'\n'影响）


				for enc in encList    : #尝试用各种encoding来解码xml文件
					try :                    
						file_xml = unicode(file_xml, encoding = enc).encode(targetEnc) #先用enc解码成unicode，再编码成targetEnc
						dom = parseString(file_xml)#把string转成xml
						print "*********************__Encoding: %s " % (enc)
						logging.info("*********************__Encoding: %s " % (enc))
						break
					except :
						pass 
				

				# 根据dom的节点读取需要提供给ECC模块的content，并交给ECC，得到bePushedObject对象
				#getElementsByTagName("TRS")获取节点
				RECs = dom.getElementsByTagName("TRS")[0].getElementsByTagName("REC")
					
				for REC in RECs:
					#判断是 新闻 or 微博，并获得content（content为即将发给ECC模型的文本内容）
					if REC.getElementsByTagName('IR_URLCONTENT').length > 0 : #节点名为IR_URLCONTENT，即新闻内容(news_QX.xml)

						# csv需要的属性：
						urlContent = REC.getElementsByTagName('IR_URLNAME')[0].childNodes[1].nodeValue # url地址
						infoTypeContent = '0' # 0代表新闻
						titleContent = REC.getElementsByTagName('IR_URLTITLE')[0].childNodes[1].nodeValue # 新闻有title
						releaseDateContent = REC.getElementsByTagName('IR_URLTIME')[0].childNodes[1].nodeValue # 发布时间（新闻）
						forwardTimesContent = '' # 新闻没有转发次数
						captureDateContent = REC.getElementsByTagName('IR_LASTTIME')[0].childNodes[1].nodeValue # 爬数据时间
							
						contentNode = REC.getElementsByTagName('IR_URLCONTENT')[0] # 正文内容，会传给ECC模块来解析
							
							
					else : #节点名为IR_STATUS_CONTENT，即微博内容(weibo_QX.xml)

						# csv需要的属性：
						urlContent = REC.getElementsByTagName('IR_URLNAME')[0].childNodes[1].nodeValue # url地址                        
						infoTypeContent = '1' # 1代表微博
						titleContent = '' # 微博没有title
						releaseDateContent = REC.getElementsByTagName('IR_CREATED_AT')[0].childNodes[1].nodeValue # 发布时间（微博）
							
						if REC.getElementsByTagName('IR_RTTCOUNT').length > 0 : # 如果没有被转发过，就不会有IR_RTTCOUNT节点，所以要进行判断
							forwardTimesContent = REC.getElementsByTagName('IR_RTTCOUNT')[0].childNodes[1].nodeValue # 微博的转发次数
						else :
							forwardTimesContent = '0'

						captureDateContent = REC.getElementsByTagName('IR_LASTTIME')[0].childNodes[1].nodeValue # 爬数据时间

							
						contentNode = REC.getElementsByTagName('IR_STATUS_CONTENT')[0]  # 正文内容，会传给ECC模块来解析

							

					myContent = ''
					for line in contentNode.childNodes    :                      
						try:
							if line.nodeType == line.TEXT_NODE or line.nodeType == line.CDATA_SECTION_NODE:
								#print line.nodeValue
								myContent = myContent + line.nodeValue.replace("\n", " ") # .replace("\n", " ")过滤回车，方便后面csv文件的观看
								pass
								#print line.nodeValue
						except Exception,e:
							pass # 如果解析有错，则吞错


						  
					print myContent.encode("gbk", "ignore")

            except Exception,e:
                print traceback.format_exc()
                logging.error(traceback.format_exc())
                     



if __name__=="__main__":
    
    print '###################################  Start!  ##########################################'
    logging.info('###################################  Start!  ##########################################')
    
    parseAndPush = ParseAndPush()
    bePushedObjectDict4CSV = parseAndPush.parseXML()

    print '##################################  End! ################################################'
    logging.info('##################################  End! ################################################')
    

              
